const puppeteer = require('puppeteer-core');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const config = require('./config');
const { checkRobotsTxt, log } = require('./utils');

async function fetchNews(url) {
  let browser;
  try {
    // 检查robots.txt
    const robotsTxt = await checkRobotsTxt(url);
    if (robotsTxt) {
      log(`Robots.txt content:\n${robotsTxt}`);
    }

    // 启动浏览器
    let executablePath = config.crawler.browserPath;
    
    // 如果未配置browserPath，尝试自动检测
    if (!executablePath) {
      const browserPaths = [
        // Chrome路径
        'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
        'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
        // Edge路径
        'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe'
      ];
      
      for (const path of browserPaths) {
        if (fs.existsSync(path)) {
          executablePath = path;
          break;
        }
      }
    }

    if (!executablePath) {
      throw new Error('未找到浏览器，请安装Chrome/Edge或在config.js中配置browserPath');
    }

    browser = await puppeteer.launch({
      executablePath,
      headless: true,
      args: ['--no-sandbox', '--disable-setuid-sandbox']
    });
    const page = await browser.newPage();
    
    // 设置用户代理
    await page.setUserAgent(config.crawler.userAgent);
    
    // 访问目标页面
    await page.goto(url, {
      waitUntil: 'networkidle2',
      timeout: config.crawler.timeout
    });
    
    // 等待新闻列表加载，增加超时时间并添加备用选择器
    await page.waitForSelector('.main-content, .ej_content.clearfix', { 
      timeout: 30000, // 延长至30秒
      visible: true  // 确保元素可见
    });
    
    // 获取渲染后的页面内容
    const newsList = await page.evaluate(() => {
      const items = [];
      // 尝试多种选择器获取新闻
      const newsContainers = [
        ...document.querySelectorAll('.main-content .news-list li'),
        ...document.querySelectorAll('.ej_content.clearfix .ej_box .ej_list li')
      ];
      
      newsContainers.forEach(element => {
        const titleEl = element.querySelector('a');
        const linkEl = element.querySelector('a');
        const timeEl = element.querySelector('span');
        
        const newsItem = {
          title: titleEl ? titleEl.textContent.trim() : '',
          publishTime: timeEl ? timeEl.textContent.trim() : '',
          sourceUrl: linkEl ? linkEl.href : '',
          images: []
        };
        
        // 处理相对路径图片
        if (linkEl && linkEl.href) {
          const baseUrl = new URL(linkEl.href).origin;
          newsItem.images.forEach(img => {
            if (img.url && !img.url.startsWith('http')) {
              img.url = baseUrl + img.url;
            }
          });
        }
        
        // 提取图片
        element.querySelectorAll('img').forEach(img => {
          if (img.src) {
            newsItem.images.push({
              url: img.src,
              localPath: null
            });
          }
        });
        
        items.push(newsItem);
      });
      return items;
    });
    
    // 模拟滚动确保所有内容加载
    await autoScroll(page);
    
    // 下载图片
    for (const news of newsList) {
      await downloadImages(news, page);
    }
    
    return newsList;
  } catch (error) {
    log(`爬取新闻失败: ${error.message}`, 'error');
    throw error;
  } finally {
    if (browser) {
      await browser.close();
    }
  }
}

async function fetchWithRetry(url, retryCount = 0) {
  try {
    const response = await axios.get(url, {
      headers: {
        'User-Agent': config.crawler.userAgent
      },
      timeout: config.crawler.timeout
    });
    return response.data;
  } catch (error) {
    if (retryCount < config.crawler.retryTimes) {
      await new Promise(resolve => setTimeout(resolve, config.crawler.delay));
      return fetchWithRetry(url, retryCount + 1);
    }
    throw error;
  }
}

async function autoScroll(page) {
  await page.evaluate(async () => {
    await new Promise((resolve) => {
      let totalHeight = 0;
      const distance = 100;
      const timer = setInterval(() => {
        const scrollHeight = document.body.scrollHeight;
        window.scrollBy(0, distance);
        totalHeight += distance;
        
        if (totalHeight >= scrollHeight) {
          clearInterval(timer);
          resolve();
        }
      }, 100);
    });
  });
}

async function downloadImages(newsItem, page) {
  // 确保图片目录存在
  if (!fs.existsSync(config.storage.imageDir)) {
    fs.mkdirSync(config.storage.imageDir, { recursive: true });
  }
  
  for (const image of newsItem.images) {
    try {
      // 使用Puppeteer访问图片URL
      const imagePage = await page.browser().newPage();
      await imagePage.goto(image.url, { waitUntil: 'networkidle2' });
      
      // 获取图片内容
      const imageBuffer = await imagePage.screenshot({ 
        encoding: 'binary',
        fullPage: true
      });
      
      // 生成文件名
      const hash = crypto.createHash('md5').update(imageBuffer).digest('hex');
      const timestamp = Date.now();
      const filename = config.storage.imageNameFormat
        .replace('{id}', newsItem.id || 'temp')
        .replace('{timestamp}', timestamp)
        .replace('{hash}', hash.substring(0, 8));
      
      // 保存图片
      const filePath = path.join(config.storage.imageDir, filename);
      fs.writeFileSync(filePath, imageBuffer);
      
      image.localPath = filePath;
      await imagePage.close();
    } catch (error) {
      log(`下载图片失败: ${image.url} - ${error.message}`, 'error');
    }
  }
}

module.exports = {
  fetchNews
};
